{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from time import time\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "from sklearn.ensemble import IsolationForest\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.datasets import fetch_kddcup99\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = fetch_kddcup99(subset=None, shuffle=True, percent10=True)\n",
    "# http://www.kdd.org/kdd-cup/view/kdd-cup-1999/Tasks\n",
    "X = dataset.data\n",
    "y = dataset.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_cols = ['duration', 'protocol_type', 'service', 'flag', 'src_bytes', 'dst_bytes', 'land', 'wrong_fragment', 'urgent', 'hot', 'num_failed_logins', 'logged_in', 'num_compromised', 'root_shell', 'su_attempted', 'num_root', 'num_file_creations', 'num_shells', 'num_access_files', 'num_outbound_cmds', 'is_host_login', 'is_guest_login', 'count', 'srv_count', 'serror_rate', 'srv_serrer_rate', 'rerror_rate', 'srv_rerror_rate', 'same_srv_rate', 'diff_srv_rate', 'srv_diff_host_rate', 'dst_host_count', 'dst_host_srv_count', 'dst_host_same_srv_rate', 'dst_host_diff_srv_rate', 'dst_host_same_src_port_rate', 'dst_host_srv_diff_host_rate', 'dst_host_serror_rate', 'dst_host_srv_serror_rate', 'dst_host_rerror_rate', 'dst_host_srv_rerror_rate']\n",
    "X = pd.DataFrame(X, columns = feature_cols)\n",
    "\n",
    "y = pd.Series(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>protocol_type</th>\n",
       "      <th>service</th>\n",
       "      <th>flag</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_host_count</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>icmp</td>\n",
       "      <td>ecr_i</td>\n",
       "      <td>SF</td>\n",
       "      <td>520</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>icmp</td>\n",
       "      <td>ecr_i</td>\n",
       "      <td>SF</td>\n",
       "      <td>1032</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>icmp</td>\n",
       "      <td>ecr_i</td>\n",
       "      <td>SF</td>\n",
       "      <td>1032</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>icmp</td>\n",
       "      <td>ecr_i</td>\n",
       "      <td>SF</td>\n",
       "      <td>1032</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>icmp</td>\n",
       "      <td>ecr_i</td>\n",
       "      <td>SF</td>\n",
       "      <td>1032</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  duration protocol_type service flag src_bytes dst_bytes land wrong_fragment  \\\n",
       "0        0          icmp   ecr_i   SF       520         0    0              0   \n",
       "1        0          icmp   ecr_i   SF      1032         0    0              0   \n",
       "2        0          icmp   ecr_i   SF      1032         0    0              0   \n",
       "3        0          icmp   ecr_i   SF      1032         0    0              0   \n",
       "4        0          icmp   ecr_i   SF      1032         0    0              0   \n",
       "\n",
       "  urgent hot           ...            dst_host_count dst_host_srv_count  \\\n",
       "0      0   0           ...                       255                255   \n",
       "1      0   0           ...                       255                255   \n",
       "2      0   0           ...                       255                255   \n",
       "3      0   0           ...                       255                255   \n",
       "4      0   0           ...                       255                255   \n",
       "\n",
       "  dst_host_same_srv_rate dst_host_diff_srv_rate dst_host_same_src_port_rate  \\\n",
       "0                      1                      0                           1   \n",
       "1                      1                      0                           1   \n",
       "2                      1                      0                           1   \n",
       "3                      1                      0                           1   \n",
       "4                      1                      0                           1   \n",
       "\n",
       "  dst_host_srv_diff_host_rate dst_host_serror_rate dst_host_srv_serror_rate  \\\n",
       "0                           0                    0                        0   \n",
       "1                           0                    0                        0   \n",
       "2                           0                    0                        0   \n",
       "3                           0                    0                        0   \n",
       "4                           0                    0                        0   \n",
       "\n",
       "  dst_host_rerror_rate dst_host_srv_rerror_rate  \n",
       "0                    0                        0  \n",
       "1                    0                        0  \n",
       "2                    0                        0  \n",
       "3                    0                        0  \n",
       "4                    0                        0  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in X.columns:  # turn whatever columns into floats that we can\n",
    "    try:\n",
    "        X[col] = X[col].astype(float)\n",
    "    except ValueError:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = pd.get_dummies(X, prefix=['protocol_type_', 'service_', 'flag_'], drop_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>...</th>\n",
       "      <th>flag__REJ</th>\n",
       "      <th>flag__RSTO</th>\n",
       "      <th>flag__RSTOS0</th>\n",
       "      <th>flag__RSTR</th>\n",
       "      <th>flag__S0</th>\n",
       "      <th>flag__S1</th>\n",
       "      <th>flag__S2</th>\n",
       "      <th>flag__S3</th>\n",
       "      <th>flag__SF</th>\n",
       "      <th>flag__SH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>520.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1032.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1032.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1032.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1032.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 115 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration  src_bytes  dst_bytes  land  wrong_fragment  urgent  hot  \\\n",
       "0       0.0      520.0        0.0   0.0             0.0     0.0  0.0   \n",
       "1       0.0     1032.0        0.0   0.0             0.0     0.0  0.0   \n",
       "2       0.0     1032.0        0.0   0.0             0.0     0.0  0.0   \n",
       "3       0.0     1032.0        0.0   0.0             0.0     0.0  0.0   \n",
       "4       0.0     1032.0        0.0   0.0             0.0     0.0  0.0   \n",
       "\n",
       "   num_failed_logins  logged_in  num_compromised    ...     flag__REJ  \\\n",
       "0                0.0        0.0              0.0    ...             0   \n",
       "1                0.0        0.0              0.0    ...             0   \n",
       "2                0.0        0.0              0.0    ...             0   \n",
       "3                0.0        0.0              0.0    ...             0   \n",
       "4                0.0        0.0              0.0    ...             0   \n",
       "\n",
       "   flag__RSTO  flag__RSTOS0  flag__RSTR  flag__S0  flag__S1  flag__S2  \\\n",
       "0           0             0           0         0         0         0   \n",
       "1           0             0           0         0         0         0   \n",
       "2           0             0           0         0         0         0   \n",
       "3           0             0           0         0         0         0   \n",
       "4           0             0           0         0         0         0   \n",
       "\n",
       "   flag__S3  flag__SF  flag__SH  \n",
       "0         0         1         0  \n",
       "1         0         1         0  \n",
       "2         0         1         0  \n",
       "3         0         1         0  \n",
       "4         0         1         0  \n",
       "\n",
       "[5 rows x 115 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "smurf.              280790\n",
       "neptune.            107201\n",
       "normal.              97278\n",
       "back.                 2203\n",
       "satan.                1589\n",
       "ipsweep.              1247\n",
       "portsweep.            1040\n",
       "warezclient.          1020\n",
       "teardrop.              979\n",
       "pod.                   264\n",
       "nmap.                  231\n",
       "guess_passwd.           53\n",
       "buffer_overflow.        30\n",
       "land.                   21\n",
       "warezmaster.            20\n",
       "imap.                   12\n",
       "rootkit.                10\n",
       "loadmodule.              9\n",
       "ftp_write.               8\n",
       "multihop.                7\n",
       "phf.                     4\n",
       "perl.                    3\n",
       "spy.                     2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9955204407492013\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=7,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fit a classification tree with max_depth=3 on all data\n",
    "from sklearn.tree import DecisionTreeClassifier, export_graphviz\n",
    "\n",
    "treeclf = DecisionTreeClassifier(max_depth=7)\n",
    "\n",
    "scores = cross_val_score(treeclf, X, y, scoring='accuracy', cv=5)\n",
    "\n",
    "print np.mean(scores)\n",
    "\n",
    "treeclf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a Graphviz file\n",
    "export_graphviz(treeclf, out_file='tree_kdd.dot', feature_names=X.columns)\n",
    "\n",
    "# At the command line, run this to convert to PNG:\n",
    "#   dot -Tpng tree_kdd.dot -o tree_kdd.png"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"tree_kdd.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>srv_count</td>\n",
       "      <td>0.633722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>same_srv_rate</td>\n",
       "      <td>0.341769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>num_compromised</td>\n",
       "      <td>0.013613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>dst_host_diff_srv_rate</td>\n",
       "      <td>0.010738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>src_bytes</td>\n",
       "      <td>0.000158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>service__red_i</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>service__private</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>service__printer</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>service__pop_3</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>service__netstat</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   feature  importance\n",
       "20               srv_count    0.633722\n",
       "25           same_srv_rate    0.341769\n",
       "9          num_compromised    0.013613\n",
       "31  dst_host_diff_srv_rate    0.010738\n",
       "1                src_bytes    0.000158\n",
       "85          service__red_i    0.000000\n",
       "84        service__private    0.000000\n",
       "83        service__printer    0.000000\n",
       "82          service__pop_3    0.000000\n",
       "75        service__netstat    0.000000"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({'feature':X.columns, 'importance':treeclf.feature_importances_}).sort_values('importance', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9997307783262454\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How about a Random Forest?\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier()\n",
    "\n",
    "scores = cross_val_score(rf, X, y, scoring='accuracy', cv=5)\n",
    "\n",
    "print np.mean(scores)  # nicer\n",
    "\n",
    "rf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>service__ecr_i</td>\n",
       "      <td>0.278599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>same_srv_rate</td>\n",
       "      <td>0.129464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>srv_count</td>\n",
       "      <td>0.108782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>src_bytes</td>\n",
       "      <td>0.101766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>flag__SF</td>\n",
       "      <td>0.073368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>flag__S0</td>\n",
       "      <td>0.058412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>count</td>\n",
       "      <td>0.055665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>dst_host_srv_count</td>\n",
       "      <td>0.038069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>protocol_type__tcp</td>\n",
       "      <td>0.036816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>dst_host_same_srv_rate</td>\n",
       "      <td>0.026287</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    feature  importance\n",
       "53           service__ecr_i    0.278599\n",
       "25            same_srv_rate    0.129464\n",
       "20                srv_count    0.108782\n",
       "1                 src_bytes    0.101766\n",
       "113                flag__SF    0.073368\n",
       "109                flag__S0    0.058412\n",
       "19                    count    0.055665\n",
       "29       dst_host_srv_count    0.038069\n",
       "38       protocol_type__tcp    0.036816\n",
       "30   dst_host_same_srv_rate    0.026287"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# more spread out importances than a single decision tree\n",
    "pd.DataFrame({'feature':X.columns, 'importance':rf.feature_importances_}).sort_values('importance', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anomoly Detection\n",
    "“An outlier is an observation in a data set which appears to be inconsistent with the remainder of that set of data.”\n",
    "\n",
    "- Supervised Anomoly Detection\n",
    "\n",
    "    - Labels available for both normal data and anomalies\n",
    "    - Similar to rare class mining / imbalanced classification\n",
    "\n",
    "- Unsupervised Anomoly Detection (Outlier Detection)\n",
    "    - no labels, training set = normal + abnormal data • Assumption: anomalies are very rare\n",
    "\n",
    "- Semi-supervised Anomoly Detection (Novelty Detection)\n",
    "    - Only normal data available to train\n",
    "    - The algorithm learns on normal data only\n",
    "\n",
    "## Isolation Forest\n",
    "\n",
    "The IsolationForest ‘isolates’ observations by randomly selecting a feature and then randomly selecting a split value between the maximum and minimum values of the selected feature.\n",
    "\n",
    "Since recursive partitioning can be represented by a tree structure, the number of splittings required to isolate a sample is equivalent to the path length from the root node to the terminating node.\n",
    "\n",
    "This path length, averaged over a forest of such random trees, is a measure of normality and our decision function.\n",
    "\n",
    "Random partitioning produces noticeably shorter paths for anomalies. Hence, when a forest of random trees collectively produce shorter path lengths for particular samples, they are highly likely to be anomalies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Supervised and Outlier Detection with KDD\n",
    "\n",
    "# In this example, we will want to use binary data where 1 will represent a \"not-normal\" attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "y_binary = y != 'normal.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    True\n",
       "1    True\n",
       "2    True\n",
       "3    True\n",
       "4    True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_binary.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y_binary)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True     0.803524\n",
       "False    0.196476\n",
       "dtype: float64"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.value_counts(normalize=True)  # check our null accuracy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IsolationForest(bootstrap=False, contamination=0.1, max_features=1.0,\n",
       "        max_samples='auto', n_estimators=100, n_jobs=1, random_state=None,\n",
       "        verbose=0)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = IsolationForest()\n",
    "model.fit(X_train)  # notice that there is no y in the .fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 1    111221\n",
       "-1     12285\n",
       "dtype: int64"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_predicted = model.predict(X_test)\n",
    "\n",
    "pd.Series(y_predicted).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    111221\n",
       "0     12285\n",
       "dtype: int64"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_predicted = np.where(y_predicted==1, 1, 0)  # turn into 0s and 1s\n",
    "\n",
    "pd.Series(y_predicted).value_counts()  # that's better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.06897078,  0.02709447,  0.16750811, ..., -0.02889957,\n",
       "       -0.0291526 ,  0.09928597])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = model.decision_function(X_test)\n",
    "\n",
    "scores  # the smaller, the more anomolous"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a2877a290>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFMpJREFUeJzt3X+MXeV95/H3t3YgFpMEE2Bq2XTtKN5qSbzNj1lAjVrNhq5toK2RCpIjFEzKylKWrFrJq43ZaIVKikSqsOwipYms4mDa7RI23SxWcOr1Qu62lUICzg8ch2U9IW6Y2g2bGCiTtESTfvvHfVzdzHNn5s6dmXvPmPdLurrnPOc5Z57vuTPzmfNj7o3MRJKkTj8z7AFIkprHcJAkVQwHSVLFcJAkVQwHSVLFcJAkVQwHSVLFcJAkVQwHSVJl9bAH0K+LL744L7nkEi644IJhD2XRfvjDH1pHg1hHs5wrdcDwazl69Oj3M/OSXvqu2HDYuHEjH//4xxkfHx/2UBat1WpZR4NYR7OcK3XA8GuJiL/sta+nlSRJFcNBklQxHCRJFcNBklQxHCRJFcNBklQxHCRJFcNBklQxHCRJlRX7H9KSNEwb9z664HX2bJnmlj7W63Ty7usWtX6vPHKQJFUMB0lSxXCQJFUMB0lSxXCQJFUMB0lSxXCQJFUMB0lSxXCQJFUMB0lSxXCQJFUMB0lSxXCQJFV6CoeIOBkRxyLi6xHxVGm7KCKORMSJ8ry2tEdE3BcRExHxdES8q2M7u0r/ExGxq6P93WX7E2XdWOpCJUm9W8iRw7/MzHdk5liZ3ws8lpmbgcfKPMA1wOby2A18EtphAtwBXAlcAdxxNlBKn90d623vuyJJ0qIt5rTSDuBAmT4AXN/R/mC2PQFcGBHrgG3Akcw8k5kvAkeA7WXZGzPzS5mZwIMd25IkDUGv4ZDA/4qIoxGxu7SNZuZpgPJ8aWlfDzzfse5kaZurfbJLuyRpSHr9JLj3ZOapiLgUOBIR/3eOvt2uF2Qf7fWG28G0G2B0dJSpqSlardacA18JrKNZrKNZmlrHni3TC15ndE1/63Ua1L7oKRwy81R5fiEiPkf7msH3ImJdZp4up4ZeKN0ngcs6Vt8AnCrt4zPaW6V9Q5f+3caxD9gHMDY2liMjI4yPj3fruqK0Wi3raBDraJam1tHPx33u2TLNPccW9+nMJ28aX9T6vZr3tFJEXBARbzg7DWwFvgkcBM7ecbQLeKRMHwRuLnctXQW8XE47HQa2RsTaciF6K3C4LHslIq4qdynd3LEtSdIQ9BJho8Dnyt2lq4E/zsw/jYgngYcj4lbgu8CNpf8h4FpgAvgR8AGAzDwTER8Fniz97szMM2X6g8ADwBrgC+UhSRqSecMhM58DfqFL+w+Aq7u0J3DbLNvaD+zv0v4U8PYexitJGgD/Q1qSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkVw0GSVDEcJEkVw0GSVOk5HCJiVUR8LSI+X+Y3RcSXI+JERHwmIs4r7eeX+YmyfGPHNm4v7c9GxLaO9u2lbSIi9i5deZKkfizkyOG3gGc65j8G3JuZm4EXgVtL+63Ai5n5VuDe0o+IuBzYCbwN2A78fgmcVcAngGuAy4H3lb6SpCHpKRwiYgNwHfAHZT6A9wKfLV0OANeX6R1lnrL86tJ/B/BQZr6amd8BJoArymMiM5/LzB8DD5W+kqQhWd1jv/8M/HvgDWX+zcBLmTld5ieB9WV6PfA8QGZOR8TLpf964ImObXau8/yM9iu7DSIidgO7AUZHR5mamqLVavVYQnNZR7NYR7M0tY49W6bn7zTD6Jr+1us0qH0xbzhExK8CL2Tm0YgYP9vcpWvOs2y29m5HL9mljczcB+wDGBsby5GREcbHx7t1XVFarZZ1NIh1NEtT67hl76MLXmfPlmnuOdbr3+TdnbxpfFHr96qXUb4H+PWIuBZ4PfBG2kcSF0bE6nL0sAE4VfpPApcBkxGxGngTcKaj/azOdWZrlyQNwbzXHDLz9szckJkbaV9QfjwzbwK+CNxQuu0CHinTB8s8ZfnjmZmlfWe5m2kTsBn4CvAksLnc/XRe+RoHl6Q6SVJfFnN882HgoYj4XeBrwP2l/X7gDyNigvYRw06AzDweEQ8D3wKmgdsy8ycAEfEh4DCwCtifmccXMS5J0iItKBwyswW0yvRztO80mtnn74AbZ1n/LuCuLu2HgEMLGYskafn4H9KSpIrhIEmqGA6SpIrhIEmqGA6SpIrhIEmqGA6SpIrhIEmqGA6SpIrhIEmqGA6SpIrhIEmqGA6SpIrhIEmqGA6SpIrhIEmqGA6SpIrhIEmqGA6SpIrhIEmqGA6SpIrhIEmqGA6SpIrhIEmqGA6SpIrhIEmqGA6SpIrhIEmqGA6SpMq84RARr4+Ir0TENyLieET8TmnfFBFfjogTEfGZiDivtJ9f5ifK8o0d27q9tD8bEds62reXtomI2Lv0ZUqSFqKXI4dXgfdm5i8A7wC2R8RVwMeAezNzM/AicGvpfyvwYma+Fbi39CMiLgd2Am8DtgO/HxGrImIV8AngGuBy4H2lryRpSOYNh2ybKrOvK48E3gt8trQfAK4v0zvKPGX51RERpf2hzHw1M78DTABXlMdEZj6XmT8GHip9JUlDsrqXTuWv+6PAW2n/lf9t4KXMnC5dJoH1ZXo98DxAZk5HxMvAm0v7Ex2b7Vzn+RntV84yjt3AboDR0VGmpqZotVq9lNBo1tEs1tEsTa1jz5bp+TvNMLqmv/U6DWpf9BQOmfkT4B0RcSHwOeCfdetWnmOWZbO1dzt6yS5tZOY+YB/A2NhYjoyMMD4+PvfgV4BWq2UdDWIdzdLUOm7Z++iC19mzZZp7jvX0a3dWJ28aX9T6vVrQ3UqZ+RLQAq4CLoyIs1VuAE6V6UngMoCy/E3Amc72GevM1i5JGpJe7la6pBwxEBFrgF8BngG+CNxQuu0CHinTB8s8ZfnjmZmlfWe5m2kTsBn4CvAksLnc/XQe7YvWB5eiOElSf3o5vlkHHCjXHX4GeDgzPx8R3wIeiojfBb4G3F/63w/8YURM0D5i2AmQmccj4mHgW8A0cFs5XUVEfAg4DKwC9mfm8SWrUJK0YPOGQ2Y+DbyzS/tztO80mtn+d8CNs2zrLuCuLu2HgEM9jFeSNAD+h7QkqWI4SJIqhoMkqWI4SJIqhoMkqWI4SJIqhoMkqWI4SJIqhoMkqWI4SJIqhoMkqWI4SJIqhoMkqWI4SJIqhoMkqWI4SJIqhoMkqWI4SJIqhoMkqWI4SJIqhoMkqWI4SJIqhoMkqWI4SJIqhoMkqWI4SJIqhoMkqWI4SJIq84ZDRFwWEV+MiGci4nhE/FZpvygijkTEifK8trRHRNwXERMR8XREvKtjW7tK/xMRsauj/d0Rcaysc19ExHIUK0nqzeoe+kwDezLzqxHxBuBoRBwBbgEey8y7I2IvsBf4MHANsLk8rgQ+CVwZERcBdwBjQJbtHMzMF0uf3cATwCFgO/CFpStTOvdt3PvoT83v2TLNLTPalsvJu68byNfR4Mx75JCZpzPzq2X6FeAZYD2wAzhQuh0Ari/TO4AHs+0J4MKIWAdsA45k5pkSCEeA7WXZGzPzS5mZwIMd25IkDcGCrjlExEbgncCXgdHMPA3tAAEuLd3WA893rDZZ2uZqn+zSLkkakl5OKwEQESPAnwC/nZl/M8dlgW4Lso/2bmPYTfv0E6Ojo0xNTdFqteYZefNZR7Os1Dr2bJn+qfnRNXXbclnO/dXU16OffbsUr8mg9kVP4RARr6MdDP81M/9Haf5eRKzLzNPl1NALpX0SuKxj9Q3AqdI+PqO9Vdo3dOlfycx9wD6AsbGxHBkZYXx8vFvXFaXVallHg6zUOmZeX9izZZp7jvX899+inLxpfNm23dTXo5/rOUvxmiznvu7Uy91KAdwPPJOZ/6lj0UHg7B1Hu4BHOtpvLnctXQW8XE47HQa2RsTacmfTVuBwWfZKRFxVvtbNHduSJA1BLxH2HuD9wLGI+Hpp+w/A3cDDEXEr8F3gxrLsEHAtMAH8CPgAQGaeiYiPAk+Wfndm5pky/UHgAWAN7buUvFNJkoZo3nDIzL+g+3UBgKu79E/gtlm2tR/Y36X9KeDt841FkjQY/oe0JKliOEiSKoaDJKliOEiSKoaDJKliOEiSKoaDJKliOEiSKoaDJKliOEiSKoaDJKliOEiSKoaDJKliOEiSKoaDJKliOEiSKoaDJKkymE8fl3RO27j30WXb9p4t09wyy/ZP3n3dsn3d1zqPHCRJFcNBklQxHCRJFcNBklQxHCRJFcNBklQxHCRJFcNBklQxHCRJFcNBklQxHCRJlXnDISL2R8QLEfHNjraLIuJIRJwoz2tLe0TEfRExERFPR8S7OtbZVfqfiIhdHe3vjohjZZ37IiKWukhJ0sL0cuTwALB9Rtte4LHM3Aw8VuYBrgE2l8du4JPQDhPgDuBK4ArgjrOBUvrs7lhv5teSJA3YvOGQmX8GnJnRvAM4UKYPANd3tD+YbU8AF0bEOmAbcCQzz2Tmi8ARYHtZ9sbM/FJmJvBgx7YkSUPS7zWH0cw8DVCeLy3t64HnO/pNlra52ie7tEuShmipP8+h2/WC7KO9+8YjdtM+BcXo6ChTU1O0Wq0+htks1tEsK7WOPVumf2p+dE3dthLNVccwX6d+9u1SvCaDqrnfcPheRKzLzNPl1NALpX0SuKyj3wbgVGkfn9HeKu0buvTvKjP3AfsAxsbGcmRkhPHx8dm6rxitVss6GmSl1jHzA3H2bJnmnmMr//O85qrj5E3jgx1Mh9k+gGguS/GaDKrmfk8rHQTO3nG0C3iko/3mctfSVcDL5bTTYWBrRKwtF6K3AofLslci4qpyl9LNHduSJA3JvBEWEf+N9l/9F0fEJO27ju4GHo6IW4HvAjeW7oeAa4EJ4EfABwAy80xEfBR4svS7MzPPXuT+IO07otYAXygPSdIQzRsOmfm+WRZd3aVvArfNsp39wP4u7U8Bb59vHJKkwVn5JySlhtnYx7lo9cd9vXx8+wxJUsVwkCRVDAdJUsVwkCRVvCCtZTXMC4Yn775uaF9bWuk8cpAkVQwHSVLFcJAkVQwHSVLFcJAkVQwHSVLFcJAkVQwHSVLFcJAkVfwPaZ2z+v3v7D1bpvv6CEjpXOKRgySpYjhIkiqGgySpYjhIkiqGgySpYjhIkiqGgySp4v85DNBs990P4r56PxVN0kJ45CBJqhgOkqSK4SBJqhgOkqSKF6RfI/p9E7qF8A3rpHNHY44cImJ7RDwbERMRsXfY45Gk17JGHDlExCrgE8C/AiaBJyPiYGZ+azm+3iD+ipaklawpRw5XABOZ+Vxm/hh4CNgx5DFJ0mtWU8JhPfB8x/xkaZMkDUFk5rDHQETcCGzLzH9d5t8PXJGZ/3ZGv93A7jL788APgO8PcqzL5GKso0mso1nOlTpg+LX8k8y8pJeOjbjmQPtI4bKO+Q3AqZmdMnMfsO/sfEQ8lZljyz+85WUdzWIdzXKu1AErq5amnFZ6EtgcEZsi4jxgJ3BwyGOSpNesRhw5ZOZ0RHwIOAysAvZn5vEhD0uSXrMaEQ4AmXkIOLTA1fbN32VFsI5msY5mOVfqgBVUSyMuSEuSmqUp1xwkSQ3S+HCIiIsi4khEnCjPa2fp96cR8VJEfH5G+wMR8Z2I+Hp5vGMwI6/Gt9g6NkXEl8v6nykX7gduAXXsKn1ORMSujvZWeZuUs6/HpYMb/fxv0xIR55f9O1H298aOZbeX9mcjYtsgxz1Tv3VExMaI+NuO/f+pQY99xjjnq+OXI+KrETEdETfMWNb1e2wYFlnHTzpej+bciJOZjX4AvwfsLdN7gY/N0u9q4NeAz89ofwC44Ryo42FgZ5n+FPDBptYBXAQ8V57Xlum1ZVkLGBvS2FcB3wbeApwHfAO4fEaffwN8qkzvBD5Tpi8v/c8HNpXtrFqBdWwEvjmMcfdZx0bgnwMPdv4cz/U9tpLqKMumhv1adHs0/siB9ttoHCjTB4Dru3XKzMeAVwY1qD70XUdEBPBe4LPzrT8AvdSxDTiSmWcy80XgCLB9QOObSy9v09JZ32eBq8v+3wE8lJmvZuZ3gImyvWFYTB1NMm8dmXkyM58G/n7Guk36HltMHY21EsJhNDNPA5Tnfk5D3BURT0fEvRFx/tIOr2eLqePNwEuZOV3mh/n2Ir3UMd/boXy6HEL/xwH/wurlbVr+sU/Z3y/T3v9NeouXxdQBsCkivhYR/ycifmm5BzuHxezTlfZ6zOX1EfFURDwREcP6o6/SiFtZI+J/Az/bZdFHlmDztwN/Tftwbx/wYeDOJdhuZRnr6PYLdNluM1uCOuYa702Z+VcR8QbgT4D30z7UHoRe9uNsfQb6GsxjMXWcBn4uM38QEe8G/mdEvC0z/2apB9mDxezTlfZ6zOXnMvNURLwFeDwijmXmt5dobH1rRDhk5q/MtiwivhcR6zLzdESsA15Y4LZPl8lXI+LTwL9bxFDn+1rLVcf3gQsjYnX5K7Dr24sslSWoYxIY75jfQPtaA5n5V+X5lYj4Y9qH5IMKh17epuVsn8mIWA28CTjT47qD0ncd2T7J/SpAZh6NiG8D/xR4atlHXVvMPp31e2wIFvW9kZmnyvNzEdEC3kn7GsZQrYTTSgeBs3ci7AIeWcjK5RfY2fP21wPfXNLR9a7vOsoP9BeBs3c5LHg/LKFe6jgMbI2IteVupq3A4YhYHREXA0TE64BfZbCvRy9v09JZ3w3A42X/HwR2lruANgGbga8MaNwz9V1HRFwS7c9Pofylupn2xdxhWMzb5nT9Hlumcc6n7zrK+M8v0xcD7wGW5XNsFmzYV8Tne9A+T/oYcKI8X1Tax4A/6Oj358D/B/6WdpJvK+2PA8do/xL6I2BkhdbxFtq/jCaA/w6c3/A6frOMdQL4QGm7ADgKPA0cB/4LA77jB7gW+H+0/zL7SGm7E/j1Mv36sn8nyv5+S8e6HynrPQtcM4z9v9g6gN8o+/4bwFeBX2t4Hf+i/Bz8kPa7MB+f63tspdUB/GL5/fSN8nzrMOvofPgf0pKkyko4rSRJGjDDQZJUMRwkSRXDQZJUMRwkSRXDQZJUMRwkSRXDQZJU+QcGdQH0iQ+Q3QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.Series(scores).hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.790868459831911"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "preds = np.where(scores < 0, 0, 1)  # customize threshold\n",
    "accuracy_score(preds, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-2 0.8035237154470228\n",
      "-0.15 0.8035237154470228\n",
      "-0.1 0.8032889090408564\n",
      "-0.05 0.8189480673003741\n",
      "0 0.790868459831911\n",
      "0.05 0.7729260116917397\n"
     ]
    }
   ],
   "source": [
    "for t in (-2, -.15, -.1, -.05, 0, .05):\n",
    "    preds = np.where(scores < t, 0, 1)  # customize threshold\n",
    "    print t, accuracy_score(preds, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "## -0.05 0.816988648325 gives us better than the null accuracy, without ever needing the testing set\n",
    "# This shows how we can can achieve predictive results without labeled data\n",
    "\n",
    "\n",
    "# This is an interesting use case of novelty detection becuase generally, when given labels\n",
    "# we do not use such tactics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 3)\n"
     ]
    }
   ],
   "source": [
    "# Detecting Malicious Urls\n",
    "\n",
    "# Load the data\n",
    "from urlparse import urlparse\n",
    "import pandas as pd\n",
    "urls = pd.read_json(\"../data/urls.json\")\n",
    "print urls.shape\n",
    "urls['string'] = \"http://\" + urls['string']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pred</th>\n",
       "      <th>string</th>\n",
       "      <th>truth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.574204e-05</td>\n",
       "      <td>http://startbuyingstocks.com/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.840909e-05</td>\n",
       "      <td>http://qqcvk.com/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.842080e-05</td>\n",
       "      <td>http://432parkavenue.com/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7.954729e-07</td>\n",
       "      <td>http://gamefoliant.ru/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.239338e-06</td>\n",
       "      <td>http://orka.cn/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3.043137e-04</td>\n",
       "      <td>http://media2.mercola.com/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4.107331e-37</td>\n",
       "      <td>http://ping.chartbeat.net/ping?h=sltrib.com&amp;p=...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.664497e-07</td>\n",
       "      <td>http://stephensteels.com/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.400715e-05</td>\n",
       "      <td>http://kbd-eko.pl/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2.273991e-05</td>\n",
       "      <td>http://ceskaposta.cz/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           pred                                             string  truth\n",
       "0  1.574204e-05                      http://startbuyingstocks.com/      0\n",
       "1  1.840909e-05                                  http://qqcvk.com/      0\n",
       "2  1.842080e-05                          http://432parkavenue.com/      0\n",
       "3  7.954729e-07                             http://gamefoliant.ru/      0\n",
       "4  3.239338e-06                                    http://orka.cn/      0\n",
       "5  3.043137e-04                         http://media2.mercola.com/      0\n",
       "6  4.107331e-37  http://ping.chartbeat.net/ping?h=sltrib.com&p=...      0\n",
       "7  1.664497e-07                          http://stephensteels.com/      0\n",
       "8  1.400715e-05                                 http://kbd-eko.pl/      0\n",
       "9  2.273991e-05                              http://ceskaposta.cz/      0"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "urls.head(10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = urls['string'], urls['truth']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    http://startbuyingstocks.com/\n",
       "1                http://qqcvk.com/\n",
       "2        http://432parkavenue.com/\n",
       "3           http://gamefoliant.ru/\n",
       "4                  http://orka.cn/\n",
       "Name: string, dtype: object"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()  # look at X\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.9694\n",
       "1    0.0306\n",
       "Name: truth, dtype: float64"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get our null accuracy because we are interested in prediction where 0 is not malicious\n",
    "\n",
    "y.value_counts(normalize=True)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "\n",
    "# Create a function called custom_tokenizer that takes in a string and outputs a list of tokens of the string.\n",
    "import re\n",
    "\n",
    "def custom_tokenizer(string):\n",
    "    final = []\n",
    "    tokens = [a for a in list(urlparse(string)) if a]\n",
    "    for t in tokens:\n",
    "        final.extend(re.compile(\"[.-]\").split(t))\n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['google', 'com']\n",
      "['https', 'google', 'so', 'not', 'fake', 'com', 'fake=False&seriously=True']\n"
     ]
    }
   ],
   "source": [
    "print custom_tokenizer('google.com')\n",
    "\n",
    "print custom_tokenizer('https://google-so-not-fake.com?fake=False&seriously=True')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "\n",
    "vect = CountVectorizer(tokenizer=custom_tokenizer)\n",
    "lr = LogisticRegression()\n",
    "lr_pipe = Pipeline([('vect', vect), ('model', lr)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.980002384002384"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score, GridSearchCV, train_test_split\n",
    "\n",
    "\n",
    "scores = cross_val_score(lr_pipe, X, y, cv=5)\n",
    "\n",
    "scores.mean()  # not good enough!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.981002585002585"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf_pipe = Pipeline([('vect', vect), ('model', RandomForestClassifier(n_estimators=500))])\n",
    "scores = cross_val_score(rf_pipe, X, y, cv=5)\n",
    "\n",
    "scores.mean()  # not as good"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1205    0]\n",
      " [  27   18]]\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "rf_pipe.fit(X_train, y_train)\n",
    "\n",
    "preds = rf_pipe.predict(X_test)\n",
    "print confusion_matrix(y_test, preds)  # hmmmm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "probs = rf_pipe.predict_proba(X_test)[:,1]  # get predicted probabilities of malicious-ness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1\n",
      "[[1190   15]\n",
      " [  15   30]]\n",
      "\n",
      "0.2\n",
      "[[1201    4]\n",
      " [  17   28]]\n",
      "\n",
      "0.3\n",
      "[[1204    1]\n",
      " [  22   23]]\n",
      "\n",
      "0.4\n",
      "[[1205    0]\n",
      " [  25   20]]\n",
      "\n",
      "0.5\n",
      "[[1205    0]\n",
      " [  27   18]]\n",
      "\n",
      "0.6\n",
      "[[1205    0]\n",
      " [  28   17]]\n",
      "\n",
      "0.7\n",
      "[[1205    0]\n",
      " [  29   16]]\n",
      "\n",
      "0.8\n",
      "[[1205    0]\n",
      " [  29   16]]\n",
      "\n",
      "0.9\n",
      "[[1205    0]\n",
      " [  30   15]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import numpy as np  # play with threshold to alter false positive/negative rate\n",
    "for thresh in [.1, .2, .3, .4, .5, .6, .7, .8, .9]:\n",
    "    preds = np.where(probs >= thresh, 1, 0)\n",
    "    print thresh\n",
    "    print confusion_matrix(y_test, preds)\n",
    "    print"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4439</th>\n",
       "      <td>decolider</td>\n",
       "      <td>0.051752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4345</th>\n",
       "      <td>cyde6743276hdjheuhde/dispatch/webs</td>\n",
       "      <td>0.045464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>789</th>\n",
       "      <td>/system/database/konto</td>\n",
       "      <td>0.045051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8547</th>\n",
       "      <td>verifiziren</td>\n",
       "      <td>0.044641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6968</th>\n",
       "      <td>php/</td>\n",
       "      <td>0.019684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6956</th>\n",
       "      <td>php</td>\n",
       "      <td>0.015053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5645</th>\n",
       "      <td>instantgrocer</td>\n",
       "      <td>0.014205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>381</th>\n",
       "      <td>/errors/report</td>\n",
       "      <td>0.013818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4813</th>\n",
       "      <td>exe</td>\n",
       "      <td>0.009287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>/</td>\n",
       "      <td>0.009121</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 feature  importance\n",
       "4439                           decolider    0.051752\n",
       "4345  cyde6743276hdjheuhde/dispatch/webs    0.045464\n",
       "789               /system/database/konto    0.045051\n",
       "8547                         verifiziren    0.044641\n",
       "6968                                php/    0.019684\n",
       "6956                                 php    0.015053\n",
       "5645                       instantgrocer    0.014205\n",
       "381                       /errors/report    0.013818\n",
       "4813                                 exe    0.009287\n",
       "92                                     /    0.009121"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({'feature':rf_pipe.steps[0][1].get_feature_names(), 'importance':rf_pipe.steps[-1][1].feature_importances_}).sort_values('importance', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9822017858017859\n"
     ]
    }
   ],
   "source": [
    "treeclf = DecisionTreeClassifier(max_depth=7)\n",
    "\n",
    "tree_pipe = Pipeline([('vect', vect), ('model', treeclf)])\n",
    "\n",
    "vect = CountVectorizer(tokenizer=custom_tokenizer)\n",
    "\n",
    "scores = cross_val_score(tree_pipe, X, y, scoring='accuracy', cv=5)\n",
    "\n",
    "print np.mean(scores)\n",
    "\n",
    "tree_pipe.fit(X, y)\n",
    "\n",
    "export_graphviz(tree_pipe.steps[1][1], out_file='tree_urls.dot', feature_names=tree_pipe.steps[0][1].get_feature_names())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"tree_urls.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "# eg. if a url has \"verifiziren\" in it, it is VERY likely malicious"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-109-a5984960075a>, line 3)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-109-a5984960075a>\"\u001b[0;36m, line \u001b[0;32m3\u001b[0m\n\u001b[0;31m    https://github.com/sinanuozdemir/blackhat-vegas-ml/blob/master/Fraud%20Detection.ipynb\u001b[0m\n\u001b[0m         ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "# Longer Credit Card Fraud Detection\n",
    "\n",
    "https://github.com/sinanuozdemir/blackhat-vegas-ml/blob/master/Fraud%20Detection.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
